package com.luo.jsoup;


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.luo.jsoup.sensi.SensitiveFilter;
import com.luo.pojo.blog.Blog;
import com.luo.pojo.blog.BlogComment;
import com.luo.pojo.blog.BlogCommentReply;
import com.luo.pojo.information.Information;
import com.luo.pojo.information.InformationComment;
import com.luo.pojo.information.InformationCommentReply;
import com.luo.service.blog.BlogCommentReplyService;
import com.luo.service.blog.BlogCommentService;
import com.luo.service.blog.BlogService;
import com.luo.service.information.InformationCommentReplyService;
import com.luo.service.information.InformationCommentService;
import com.luo.service.information.InformationService;
import com.luo.util.DateKit;
import com.vdurmont.emoji.EmojiParser;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.Random;
import java.util.Scanner;

/**
 * @author luowei
 * @create 2020-08-6 16:14
 *  爬虫工具类
 */
public class JsoupUtil {

    //    计时工具
    private static ThreadLocal<Long> startTime = new ThreadLocal<Long>();

    //脏字词过滤
    private static SensitiveFilter filter = SensitiveFilter.DEFAULT;

    private BlogService blogService;                                          //博客表 操作的Service，也可以指直接是mapper
    private BlogCommentService blogCommentService;                            //博客评论表 操作的Service，也可以指直接是mapper
    private BlogCommentReplyService blogCommentReplyService;                  //博客回复表 操作的Service，也可以指直接是mapper
    private InformationService informationService;                            //资讯表 操作的Service，也可以指直接是mapper
    private InformationCommentService informationCommentService;              //资讯评论表 操作的Service，也可以指直接是mapper
    private InformationCommentReplyService informationCommentReplyService;    //资讯回复表 操作的Service，也可以指直接是mapper

//    需要注入service去操作数据库,这里因为我用了mybatisPlus逆向工程，直接通过Service层的通用crud即可
    public JsoupUtil(BlogService blogService,InformationService informationService,BlogCommentService blogCommentService,BlogCommentReplyService blogCommentReplyService,InformationCommentService informationCommentService,InformationCommentReplyService informationCommentReplyService){
        this.blogService = blogService;
        this.informationService = informationService;
        this.blogCommentService = blogCommentService;
        this.blogCommentReplyService = blogCommentReplyService;
        this.informationCommentService = informationCommentService;
        this.informationCommentReplyService = informationCommentReplyService;
    }

//    爬虫程序主入口
    public void getStart(){

        String url;
        int num = 0;
        Scanner scanner = new Scanner(System.in);
        String choice = makeChoice(scanner);
        if ("1".equals(choice)){
            System.out.println("开始录入csdn博客到blog表中");
            System.out.print("请输入博客的url(键入 end 结束): ");
            while(!"end".equals(url=scanner.nextLine())){
                if (url.contains("blog.csdn.net")){
                    System.out.println("已将 "+getCSDNBlogByUrl(url)+" 录入数据库");
                    num++;
                    System.out.print("请输入博客的url(键入 end 结束): ");
                }else {
                    System.out.println("url错误!程序终止!");
                    break;
                }
            }
            if (num!=0){
                System.out.println("博客录入完毕！共录入 "+num+" 个博客！");
            }
        }
        else if ("2".equals(choice)){
            System.out.println("开始录入凤凰网资讯到information表中");
            System.out.print("请输入资讯的url(键入 end 结束): ");
            while(!"end".equals(url=scanner.nextLine())){
                if (url.contains(".ifeng.com")){
                    System.out.println("已将 "+getFengHuangInformationByUrl(url)+" 录入数据库");
                    num++;
                    System.out.print("请输入资讯的url(键入 end 结束): ");
                }else {
                    System.out.println("url错误!程序终止!");
                    break;
                }
            }
            if (num!=0) {
                System.out.println("资讯录入完毕！共录入 " + num + " 个资讯！");
            }
        }else {
            System.out.println("键入无效值，程序结束！");
        }
    }

    //选择器 1：录入博客 2：录入资讯
    public String makeChoice(Scanner scanner){
        System.out.println("请选择：1：录入博客   2：录入资讯  键入其他内容无效");
        System.out.print("请输入选择: ");
        return scanner.nextLine();
    }


    /**
     * 核心作用是计时，然后会调用底层的入库函数
     * @param url   博客的url
     * @author PRO Luo
     * @Time 2020/8/6
     * @return 如果录入完成会返回博客的标题
     *
     */
    public String getCSDNBlogByUrl(String url){
        String blogTitle = null;
        try {
            startTime.set(System.currentTimeMillis());
            blogTitle = CSDNSoup(url);
            System.out.println("用时:   "+(System.currentTimeMillis()-startTime.get())+"ms");
            startTime.remove();
        } catch (Exception e) {

        }finally {
            return blogTitle;
        }
    }


    //    获取CSDN博客信息并入库
    private String CSDNSoup(String url) throws Exception {
//        解析url
        Document document = Jsoup.parse(new URL(url), 30000);
//        选择博客内容主题部分并注入到blog实体类中
        Elements blogContent = document.select(".blog-content-box");

//        将爬取的blog数据注入到blog类中,再一并录入数据库
        Blog blog = new Blog();
        blog.setTitle(blogContent.select("#articleContentId").text());
        blog.setAuthor(blogContent.select(".bar-content").select(".follow-nickName").text());
//        emojiParse,表情解析转换
        blog.setContent(emojiParse(blogContent.select("#article_content").html()));
        blog.setInstruction("此博客转载自CSDN");

//        避免0阅读情况异常
        String readNum = blogContent.select(".bar-content").select(".read-count").text();
        if (readNum == null||"".equals(readNum)){
            blog.setReadNum(0);
        }else {
            blog.setReadNum(Integer.parseInt(readNum));
        }

//        避免0收藏情况异常
        String collectNum = blogContent.select(".bar-content").select(".get-collection").text();
        if (collectNum == null||"".equals(collectNum)){
            blog.setCollectNum(0);
        }else {
            blog.setCollectNum(Integer.parseInt(collectNum));
        }

//        随机选择一个数据库中已有的用户id，我这里user表中只有三个用户，故nextInt的bound为3
        long seed = System.nanoTime();
        Random random = new Random(seed);
        int randomUser;
        while ((randomUser =  random.nextInt(3))==0){}
        blog.setUserId(randomUser);

//        这里的DateKit是对string转date方法的封装
        String date = blogContent.select(".bar-content").select(".time").text();
        blog.setCreateTime(DateKit.dateFormat(date));

//        取得更多与博客相关的内容
        Elements moreInformation = document.select(".toolbox-list");
        String likeNum = moreInformation.select("#is-like").select("#spanCount").text();
//        避免0点赞情况异常
        if (likeNum == null || "".equals(likeNum)){
            blog.setLikeNum(0);
        }else {
            blog.setLikeNum(Integer.parseInt(likeNum));
        }

        String commentNum = moreInformation.select(".tool-item-comment").select(".count").text();
//        避免0评论情况异常
        if (commentNum == null || "".equals(commentNum)){
            blog.setCommentNum(0);
        }else {
            blog.setCommentNum(Integer.parseInt(commentNum));
        }

//        持久化blog，即入库
        blogService.save(blog);


//        因为评论是动态加载的，需要拿到对应存放评论的json的url
        int pageNo = 1;     //当前页数
        int pageCount = 2;  //总页数，为了保证第一次能运行的初值

//        循环爬取该博客的所有评论和回复，csdn分页处理故需要分页爬取
        while (true){
            if (pageNo>pageCount){
                break;
            }
//            获取含有评论的url
            String commentUrl = getCommentUrl(url, pageNo++);
//            获取json
            String json = loadJson(commentUrl);

            JSONObject jsonObject = JSON.parseObject(json);
            JSONObject  data = jsonObject.getJSONObject("data");
//            为总页数赋初值，这里是通过解析json拿到的总页数
            if (pageNo==2){
                pageCount = Integer.parseInt(data.getString("page_count"));
            }
            JSONArray CommentArray = data.getJSONArray("list");
//            每个评论的爬取
            for (int i=0;i<CommentArray.size();i++){
//                将数据注入到实体类，然后一并存入数据库
                BlogComment comment = new BlogComment();
                JSONObject object = CommentArray.getJSONObject(i);
                JSONObject commentString = object.getJSONObject("info");
                comment.setContent(emojiParse(commentString.getString("Content")));
//                过滤带脏字词or敏感词的评论
                if (filter.filter(comment.getContent(),'*').contains("*")){
                    blog.setCommentNum((blog.getCollectNum()-1));
                    continue;
                }
                comment.setUserName(commentString.getString("NickName"));
                comment.setUserAvatar(commentString.getString("Avatar"));
                comment.setLikeNum(Integer.parseInt(commentString.getString("Digg")));
                comment.setCreateTime(DateKit.dateFormat(commentString.getString("PostTime")));
                comment.setBlogId(
                    blogService.getOne(
                            new QueryWrapper<Blog>()
                                    .eq("title",blog.getTitle())
                                    .eq("author",blog.getAuthor())
                                    .eq("user_id",blog.getUserId())
                    ).getId()
                );
//                持久化评论
                blogCommentService.save(comment);
                System.out.println(comment);

                JSONArray replyArray = object.getJSONArray("sub");
                if (replyArray!=null&&replyArray.size()!=0){
                    for (int j=0;j<replyArray.size();j++){
                        BlogCommentReply reply = new BlogCommentReply();
                        JSONObject replyString = replyArray.getJSONObject(j);
        //                        除去csdn前端的标识符
                        String content = replyString.getString("Content");
                        reply.setContent(emojiParse(content.replaceAll("\\p{Punct}?reply\\p{Punct}?.*\\p{Punct}?/reply\\p{Punct}?","")));
                        //                过滤带脏字词or敏感词的回复
                        if (filter.filter(reply.getContent(),'*').contains("*")){
                            continue;
                        }
                        reply.setUserName(replyString.getString("NickName"));
                        reply.setUserAvatar(replyString.getString("Avatar"));
                        reply.setLikeNum(Integer.parseInt(replyString.getString("Digg")));
                        reply.setCreateTime(DateKit.dateFormat(replyString.getString("PostTime")));
                        reply.setReplyName(replyString.getString("ParentNickName"));
                        reply.setBlogCommentid(
                                blogCommentService.getOne(
                                    new QueryWrapper<BlogComment>()
                                            .eq("content",comment.getContent())
                                            .eq("blog_id",comment.getBlogId())
                                            .eq("user_name",comment.getUserName())
                                ).getId()
                        );
//                        持久化回复
                        blogCommentReplyService.save(reply);
                        System.out.println(reply);
                    }

                }
            }
        }
        blogService.update()
                .eq("title",blog.getTitle())
                .eq("author",blog.getAuthor())
                .eq("user_id",blog.getUserId())
                .set("comment_num",blog.getCommentNum());
        System.out.println(blog);
        return blog.getTitle();
    }

    /**
     * 通过url获取对应博客评论的json请求 通过正则表达式获取，二级域名以及用户的uid
     * @param url 博客的url
     * @param pageNo 评论分页页数
     * @return:评论的json路径
     */
    private String getCommentUrl(String url,int pageNo){

        String regex1 = "https://blog.csdn.net/";
        String regex2 = "/article/details/\\d+\\p{Punct}?.*";
        String userName = url.replaceAll(regex1, "");
        userName = userName.replaceAll(regex2,"");

        String regex3 = "https://blog.csdn.net/.*/article/details/";
        String uid = url.replaceAll(regex3,"");
        StringBuffer buffer = new StringBuffer();
        char[] chars = uid.toCharArray();

        for (int i=0;i<chars.length&&(chars[i]-'\ud802')!=-55235;i++){
            buffer.append(chars[i]);
        }
        uid = buffer.toString();

        return "https://blog.csdn.net/"+userName+"/phoenix/comment/list/"+uid+"?page="+pageNo+"&size=10&tree_type=1&comment_id=";
    }

    /**
     * 通过网络访问json并读取文件
     * @param url 博客的url
     * @return:json文件的内容
     */
    private String loadJson (String url) {
        StringBuilder json = new StringBuilder();
        try {
            URL urlObject = new URL(url);
            URLConnection uc = urlObject.openConnection();
            BufferedReader in = new BufferedReader(new InputStreamReader(uc.getInputStream(),"UTF-8"));
            String inputLine = null;
            while ( (inputLine = in.readLine()) != null) {
                json.append(inputLine);
            }
            in.close();

        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return json.toString();
    }

    /**
     *  解析文章、评论以及回复中的表情，emoji解析
     * @param content  需要解析的内容
     * @return 解析后的表情别名，如果要将content拿出数据库，也需要用emoji将别名解析成表情
     */
    private String emojiParse(String content){
        return EmojiParser.parseToAliases(content);
    }

    /**
     * 核心作用是计时，然后会调用底层的入库函数
     * @param url   资讯的url
     * @author PRO Luo
     * @time 2020/8/6
     * @return FengHuangInformation实体对象
     */
    public String getFengHuangInformationByUrl(String url){
        String information = null;
        try {
            startTime.set(System.currentTimeMillis());
            //同样可以做持久化操作
            information = FengHuangSoup(url);
            System.out.println("用时:   "+(System.currentTimeMillis()-startTime.get())+"ms");
            startTime.remove();
        } catch (Exception e) {

        }finally {
            return information;
        }
    }

    //    获取凤凰资讯网资讯信息
    private String FengHuangSoup(String url) throws Exception {
//        jsoup解析url，获得document
        Document document = Jsoup.parse(new URL(url), 30000);
        Elements informationContent = document.select("#root");

//                将数据注入到实体类，然后一并存入数据库
        Information information = new Information();
        information.setTitle(informationContent.select("h1").text());
        information.setAuthor(informationContent.select("p[class^='time']").select("span").eq(2).text());
        information.setContent(emojiParse(informationContent.select("div[class^='text']").html()));
        SimpleDateFormat format = new SimpleDateFormat("yyyy年MM月dd日 HH:mm:ss");
//        凤凰网的时间有两种格式
        try {
            information.setCreateTime(format.parse(informationContent.select("p[class^='time']").select("span").eq(0).text()));
        }catch (Exception e){
            information.setCreateTime(DateKit.dateFormat(informationContent.select("p[class^='time']").select("span").eq(0).text()));
        }
        information.setIfReprint(1);
        //        随机选择一个数据库中已有的用户id，我这里user表中只有三个用户，故nextInt的bound为3
        long seed = System.nanoTime();
        Random random = new Random(seed);
        int randomUser;
        while ((randomUser =  random.nextInt(3))==0){}
        information.setUserId(randomUser);
        information.setInstruction("资讯来自于凤凰网");


        int pageNo = 1;     //当前页数
        int pageCount = 2;  //总页数，为了保证第一次能运行的初值

        while (true){
            if (pageNo>pageCount){
                break;
            }
            String commentUrl = getInformationCommentUrl(url, pageNo++);
            String json = loadJson(commentUrl);
            json = json.replaceAll("\\p{Punct}?function\\p{Punct}?\\p{Punct}?\\p{Punct}?var commentJsonVarStr___=","");
            json = json.replaceAll("; newCommentListCallBack\\p{Punct}?commentJsonVarStr___\\p{Punct}?;\\p{Punct}?\\p{Punct}?\\p{Punct}?\\p{Punct}?;","");
            JSONObject jsonObject = JSON.parseObject(json);
//            为总页数赋初值，这里是通过解析json拿到的总页数
            if (pageNo==2){
                int totalRecord = Integer.parseInt(jsonObject.getString("count")) ;
                pageCount = (totalRecord%10==0)?(totalRecord/10):(totalRecord/10+1);

                information.setCommentNum(totalRecord);
                informationService.save(information);

            }
            JSONArray CommentArray = jsonObject.getJSONArray("comments");
//            每个评论的爬取
            for (int i=0;i<CommentArray.size();i++){
//                将数据注入到实体类，然后一并存入数据库
                InformationComment informationComment = new InformationComment();
                JSONObject comment = CommentArray.getJSONObject(i);
                informationComment.setUserName(comment.getString("uname"));
                informationComment.setContent(emojiParse(comment.getString("comment_contents")));
                //                过滤带脏字词or敏感词的评论
                if (filter.filter(informationComment.getContent(),'*').contains("*")){
                    information.setCommentNum((information.getCommentNum()-1));
                    continue;
                }
                SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm");
                informationComment.setCreateTime(dateFormat.parse(comment.getString("comment_date")));
                informationComment.setLikeNum(Integer.parseInt(comment.getString("uptimes")));
                informationComment.setInformationId(
                        informationService.getOne(
                            new QueryWrapper<Information>()
                                .eq("title",information.getTitle())
                                .eq("author",information.getAuthor())
                                .eq("user_id",information.getUserId())
                        ).getId()
                );

//                持久化评论
                informationCommentService.save(informationComment);
                System.out.println(informationComment);

                JSONArray replyArray = comment.getJSONArray("parent");
                if (replyArray!=null&&replyArray.size()!=0){
                    for (int j=0;j<replyArray.size();j++){
//                将数据注入到实体类，然后一并存入数据库
                        InformationCommentReply informationCommentReply = new InformationCommentReply();
                        JSONObject replyString = replyArray.getJSONObject(j);
                        informationCommentReply.setContent(emojiParse(replyString.getString("comment_contents")));
                        //                过滤带脏字词or敏感词的回复
                        if (filter.filter(informationCommentReply.getContent(),'*').contains("*")){
                            continue;
                        }
                        informationCommentReply.setUserName(replyString.getString("uname"));
                        informationCommentReply.setLikeNum(Integer.parseInt(replyString.getString("uptimes")));
                        informationCommentReply.setCreateTime(dateFormat.parse(replyString.getString("comment_date")));
                        informationCommentReply.setReplyName(informationComment.getUserName());
                        informationCommentReply.setInformationCommentid(
                                informationCommentService.getOne(
                                        new QueryWrapper<InformationComment>()
                                                .eq("content",informationComment.getContent())
                                                .eq("information_id",informationComment.getInformationId())
                                                .eq("user_name",informationComment.getUserName())
                                ).getId()
                        );
//                        持久化回复
                        informationCommentReplyService.save(informationCommentReply);
                        System.out.println(informationCommentReply);
                    }

                }
            }
        }
        informationService.update()
                .eq("title",information.getTitle())
                .eq("author",information.getAuthor())
                .eq("user_id",information.getUserId())
                .set("comment_num",information.getCommentNum());
        System.out.println(information);
        return information.getTitle();

    }


    /**
     * 通过url获取对应资讯评论的json请求
     * @param url 资讯的url
     * @param pageNo 评论分页页数
     * @return:评论的json路径
     */
    private String getInformationCommentUrl(String url,int pageNo){

        String informationId = url.replaceAll("https://news.ifeng.com/c/", "");
        String head = "https://comment.ifeng.com/get.php?orderby=uptimes&docUrl=ucms_";
        String middle = "&format=js&job=1&p=";
        String end = "&pageSize=10&callback=newCommentListCallBack&_=true";

        return head + informationId + middle + pageNo + end;

    }

}
